Skip to content

Commit dcf7c59

Browse files
author
Jochen Schalanda
committed
Migrate to Elasticsearch 2.1.0
1 parent 0a8944b commit dcf7c59

40 files changed

+478
-599
lines changed

‎docker/config/do.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
export PROJECT_NAME=${PROJECT_NAME:-"graylog"}
22

33
export ELASTICSEARCH_IMAGE=${ELASTICSEARCH_IMAGE:-"elasticsearch"}
4-
export ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION:-"1.7"}
4+
export ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION:-"2.1"}
55
export ELASTICSEARCH_CONFIG_VOLUME=${ELASTICSEARCH_CONFIG_VOLUME:-"./elasticsearch"}
66
export ELASTICSEARCH_DATA_VOLUME=${ELASTICSEARCH_DATA_VOLUME:-"graylog-elasticsearch"}
77
export ELASTICSEARCH_CLUSTER_NAME=${ELASTICSEARCH_CLUSTER_NAME:-"graylog"}

‎graylog2-bootstrap/pom.xml

-15
Original file line numberDiff line numberDiff line change
@@ -80,21 +80,6 @@
8080
</dependencies>
8181

8282
<build>
83-
<resources>
84-
<resource>
85-
<!-- git-commit-id-plugin depends on resource filtering to write the
86-
git.properties and version.properties files-->
87-
<directory>src/main/resources</directory>
88-
<filtering>true</filtering>
89-
<includes>
90-
<include>**/*.properties</include>
91-
</includes>
92-
</resource>
93-
<resource>
94-
<directory>src/main/resources</directory>
95-
</resource>
96-
</resources>
97-
9883
<plugins>
9984
<plugin>
10085
<groupId>org.apache.maven.plugins</groupId>

‎graylog2-server/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -325,8 +325,8 @@
325325
<artifactId>assertj-joda-time</artifactId>
326326
</dependency>
327327
<dependency>
328-
<groupId>com.lordofthejars</groupId>
329-
<artifactId>nosqlunit-elasticsearch</artifactId>
328+
<groupId>com.github.joschi.nosqlunit</groupId>
329+
<artifactId>nosqlunit-elasticsearch2</artifactId>
330330
</dependency>
331331
<dependency>
332332
<groupId>com.lordofthejars</groupId>

‎graylog2-server/src/main/java/org/elasticsearch/node/GraylogNode.java

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/**
2+
* This file is part of Graylog.
3+
*
4+
* Graylog is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* Graylog is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with Graylog. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
package org.elasticsearch.node;
18+
19+
import org.elasticsearch.Version;
20+
import org.elasticsearch.common.settings.Settings;
21+
import org.elasticsearch.plugins.Plugin;
22+
23+
import java.util.Collection;
24+
25+
/**
26+
* A custom Elasticsearch {@link Node} which allows adding plugins from the classpath.
27+
*/
28+
public final class GraylogNode extends Node {
29+
/**
30+
* Constructs a node with the given settings.
31+
*
32+
* @param preparedSettings Base settings to configure the node with
33+
* @param classpathPlugins A list of classpath plugins to load on startup
34+
*/
35+
public GraylogNode(Settings preparedSettings, Collection<Class<? extends Plugin>> classpathPlugins) {
36+
super(preparedSettings, Version.CURRENT, classpathPlugins);
37+
}
38+
}

‎graylog2-server/src/main/java/org/graylog2/ESTimestampFixup.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ private void startEsNode() {
287287

288288
private void stopEsNode() {
289289
LOG.debug("Stopping ES node");
290-
node.stop();
290+
node.close();
291291
}
292292

293293
private void processBulk(BulkRequestBuilder bulk, boolean fix) {

‎graylog2-server/src/main/java/org/graylog2/bindings/providers/EsNodeProvider.java

+39-41
Original file line numberDiff line numberDiff line change
@@ -16,27 +16,26 @@
1616
*/
1717
package org.graylog2.bindings.providers;
1818

19-
import com.google.common.base.Joiner;
20-
import com.google.common.collect.ImmutableList;
21-
import com.google.common.collect.Maps;
22-
import javax.inject.Inject;
23-
import javax.inject.Provider;
24-
import javax.inject.Singleton;
25-
import org.elasticsearch.common.settings.loader.YamlSettingsLoader;
19+
import org.elasticsearch.common.settings.Settings;
20+
import org.elasticsearch.common.settings.SettingsException;
21+
import org.elasticsearch.node.GraylogNode;
2622
import org.elasticsearch.node.Node;
27-
import org.elasticsearch.node.NodeBuilder;
23+
import org.elasticsearch.plugins.Plugin;
2824
import org.graylog2.configuration.ElasticsearchConfiguration;
25+
import org.graylog2.indexer.esplugin.MonitorPlugin;
2926
import org.slf4j.Logger;
3027
import org.slf4j.LoggerFactory;
3128

32-
import java.io.File;
33-
import java.io.IOException;
34-
import java.nio.file.Files;
35-
import java.util.Map;
29+
import javax.inject.Inject;
30+
import javax.inject.Provider;
31+
import javax.inject.Singleton;
32+
import java.nio.file.Path;
33+
import java.util.Collections;
34+
import java.util.List;
3635

3736
import static com.google.common.base.Strings.isNullOrEmpty;
38-
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
3937

38+
@Singleton
4039
public class EsNodeProvider implements Provider<Node> {
4140
private static final Logger LOG = LoggerFactory.getLogger(EsNodeProvider.class);
4241

@@ -48,43 +47,45 @@ public EsNodeProvider(ElasticsearchConfiguration configuration) {
4847
}
4948

5049
@Override
51-
@Singleton
5250
public Node get() {
53-
final NodeBuilder builder = nodeBuilder().client(configuration.isClientNode());
54-
Map<String, String> settings = readNodeSettings(configuration);
55-
56-
builder.settings().put(settings);
57-
return builder.build();
51+
return new GraylogNode(
52+
readNodeSettings(configuration),
53+
Collections.<Class<? extends Plugin>>singleton(MonitorPlugin.class));
5854
}
5955

60-
public static Map<String, String> readNodeSettings(ElasticsearchConfiguration conf) {
61-
Map<String, String> settings = Maps.newHashMap();
56+
public static Settings readNodeSettings(ElasticsearchConfiguration conf) {
57+
final Settings.Builder settings = Settings.builder();
6258

6359
// Standard Configuration.
6460
settings.put("cluster.name", conf.getClusterName());
6561

6662
settings.put("node.name", conf.getNodeName());
67-
settings.put("node.master", Boolean.toString(conf.isMasterNode()));
68-
settings.put("node.data", Boolean.toString(conf.isDataNode()));
63+
settings.put("node.master", conf.isMasterNode());
64+
settings.put("node.data", conf.isDataNode());
65+
settings.put("node.client", true);
66+
6967

68+
settings.put("path.home", conf.getPathHome());
7069
if (!isNullOrEmpty(conf.getPathData())) {
7170
settings.put("path.data", conf.getPathData());
7271
}
7372

74-
settings.put("action.auto_create_index", Boolean.toString(false));
73+
settings.put("action.auto_create_index", false);
7574

76-
settings.put("http.enabled", Boolean.toString(conf.isHttpEnabled()));
77-
settings.put("transport.tcp.port", String.valueOf(conf.getTransportTcpPort()));
75+
settings.put("http.enabled", conf.isHttpEnabled());
76+
settings.put("transport.tcp.port", conf.getTransportTcpPort());
7877

7978
settings.put("discovery.initial_state_timeout", conf.getInitialStateTimeout());
80-
settings.put("discovery.zen.ping.multicast.enabled", Boolean.toString(conf.isMulticastDiscovery()));
81-
82-
if (conf.getUnicastHosts() != null && !conf.getUnicastHosts().isEmpty()) {
83-
final ImmutableList.Builder<String> trimmedHosts = ImmutableList.builder();
84-
for (String host : conf.getUnicastHosts()) {
85-
trimmedHosts.add(host.trim());
79+
settings.put("discovery.zen.ping.multicast.enabled", conf.isMulticastDiscovery());
80+
81+
final List<String> unicastHosts = conf.getUnicastHosts();
82+
if (unicastHosts != null && !unicastHosts.isEmpty()) {
83+
final String[] trimmedHosts = new String[unicastHosts.size()];
84+
for (int i = 0; i < unicastHosts.size(); i++) {
85+
final String host = unicastHosts.get(i);
86+
trimmedHosts[i] = host.trim();
8687
}
87-
settings.put("discovery.zen.ping.unicast.hosts", Joiner.on(",").join(trimmedHosts.build()));
88+
settings.putArray("discovery.zen.ping.unicast.hosts", trimmedHosts);
8889
}
8990

9091
if (!isNullOrEmpty(conf.getNetworkHost())) {
@@ -97,19 +98,16 @@ public static Map<String, String> readNodeSettings(ElasticsearchConfiguration co
9798
settings.put("network.publish_host", conf.getNetworkPublishHost());
9899
}
99100

100-
settings.put("plugins.mandatory", "graylog2-monitor");
101-
102101
// Overwrite from a custom ElasticSearch config file.
103-
final File esConfigFile = conf.getConfigFile();
102+
final Path esConfigFile = conf.getConfigFile();
104103
if (esConfigFile != null) {
105104
try {
106-
final byte[] esSettings = Files.readAllBytes(esConfigFile.toPath());
107-
settings.putAll(new YamlSettingsLoader().load(esSettings));
108-
} catch (IOException e) {
109-
LOG.warn("Cannot read Elasticsearch configuration.");
105+
settings.loadFromPath(esConfigFile);
106+
} catch (SettingsException e) {
107+
LOG.warn("Cannot read Elasticsearch configuration from " + esConfigFile, e);
110108
}
111109
}
112110

113-
return settings;
111+
return settings.build();
114112
}
115113
}

‎graylog2-server/src/main/java/org/graylog2/configuration/ElasticsearchConfiguration.java

+12-5
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@
1919
import com.github.joschi.jadconfig.Parameter;
2020
import com.github.joschi.jadconfig.converters.StringListConverter;
2121
import com.github.joschi.jadconfig.util.Duration;
22-
import com.github.joschi.jadconfig.validators.FileReadableValidator;
22+
import com.github.joschi.jadconfig.validators.FilePathReadableValidator;
2323
import com.github.joschi.jadconfig.validators.InetPortValidator;
2424
import com.github.joschi.jadconfig.validators.PositiveDurationValidator;
2525
import com.github.joschi.jadconfig.validators.PositiveIntegerValidator;
2626
import com.github.joschi.jadconfig.validators.PositiveLongValidator;
2727
import org.joda.time.Period;
2828

29-
import java.io.File;
29+
import java.nio.file.Path;
3030
import java.util.List;
3131
import java.util.Locale;
3232

@@ -46,6 +46,9 @@ public class ElasticsearchConfiguration {
4646
@Parameter(value = "elasticsearch_path_data")
4747
private String pathData = "data/elasticsearch";
4848

49+
@Parameter(value = "elasticsearch_path_home")
50+
private String pathHome = "data/elasticsearch";
51+
4952
@Parameter(value = "elasticsearch_transport_tcp_port", validator = InetPortValidator.class)
5053
private int transportTcpPort = 9350;
5154

@@ -76,8 +79,8 @@ public class ElasticsearchConfiguration {
7679
@Parameter(value = "elasticsearch_disable_version_check")
7780
private boolean disableVersionCheck = false;
7881

79-
@Parameter(value = "elasticsearch_config_file", validator = FileReadableValidator.class)
80-
private File configFile; // = "/etc/graylog/server/elasticsearch.yml";
82+
@Parameter(value = "elasticsearch_config_file", validator = FilePathReadableValidator.class)
83+
private Path configFile; // = "/etc/graylog/server/elasticsearch.yml";
8184

8285
@Parameter(value = "elasticsearch_index_prefix", required = true)
8386
private String indexPrefix = "graylog2";
@@ -181,7 +184,7 @@ public boolean isDisableVersionCheck() {
181184
return disableVersionCheck;
182185
}
183186

184-
public File getConfigFile() {
187+
public Path getConfigFile() {
185188
return configFile;
186189
}
187190

@@ -245,6 +248,10 @@ public String getPathData() {
245248
return pathData;
246249
}
247250

251+
public String getPathHome() {
252+
return pathHome;
253+
}
254+
248255
public Duration getRequestTimeout() {
249256
return requestTimeout;
250257
}

‎graylog2-server/src/main/java/org/graylog2/indexer/IndexHelper.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818

1919
import com.google.common.collect.Lists;
2020
import com.google.common.collect.Sets;
21-
import org.elasticsearch.index.query.FilterBuilder;
22-
import org.elasticsearch.index.query.FilterBuilders;
21+
import org.elasticsearch.index.query.QueryBuilder;
22+
import org.elasticsearch.index.query.QueryBuilders;
2323
import org.graylog2.database.NotFoundException;
2424
import org.graylog2.indexer.ranges.IndexRange;
2525
import org.graylog2.indexer.ranges.IndexRangeService;
@@ -56,12 +56,12 @@ public static Set<String> getOldestIndices(Set<String> indexNames, int count) {
5656
return r;
5757
}
5858

59-
public static FilterBuilder getTimestampRangeFilter(TimeRange range) throws InvalidRangeFormatException {
59+
public static QueryBuilder getTimestampRangeFilter(TimeRange range) throws InvalidRangeFormatException {
6060
if (range == null) {
6161
return null;
6262
}
6363

64-
return FilterBuilders.rangeFilter("timestamp")
64+
return QueryBuilders.rangeQuery("timestamp")
6565
.gte(Tools.buildElasticSearchTimeFormat(range.getFrom()))
6666
.lte(Tools.buildElasticSearchTimeFormat(range.getTo()));
6767
}

‎graylog2-server/src/main/java/org/graylog2/indexer/IndexMapping.java

+15-20
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,7 @@ public Map<String, Object> messageMapping(final String analyzer) {
3737
return ImmutableMap.of(
3838
"properties", partFieldProperties(analyzer),
3939
"dynamic_templates", partDefaultAllInDynamicTemplate(),
40-
// Compress source field
41-
"_source", enabledAndCompressed(),
40+
"_source", enabled(),
4241
// Enable purging by TTL
4342
"_ttl", enabled());
4443
}
@@ -47,20 +46,16 @@ public Map<String, Object> messageMapping(final String analyzer) {
4746
* Disable analyzing for every field by default.
4847
*/
4948
private List<Map<String, Map<String, Object>>> partDefaultAllInDynamicTemplate() {
50-
final Map<String, Serializable> mappingInternal = ImmutableMap.<String, Serializable>of(
51-
"index", "not_analyzed",
52-
"doc_values", true);
5349
final Map<String, Object> defaultInternal = ImmutableMap.of(
5450
"match", "gl2_*",
55-
"mapping", mappingInternal);
51+
"mapping", notAnalyzedString());
5652
final Map<String, Map<String, Object>> templateInternal = ImmutableMap.of("internal_fields", defaultInternal);
5753

58-
final Map<String, String> mappingAll = ImmutableMap.of("index", "not_analyzed");
5954
final Map<String, Object> defaultAll = ImmutableMap.of(
6055
// Match all
6156
"match", "*",
6257
// Analyze nothing by default
63-
"mapping", mappingAll);
58+
"mapping", ImmutableMap.of("index", "not_analyzed"));
6459
final Map<String, Map<String, Object>> templateAll = ImmutableMap.of("store_generic", defaultAll);
6560

6661
return ImmutableList.of(templateInternal, templateAll);
@@ -75,9 +70,16 @@ private List<Map<String, Map<String, Object>>> partDefaultAllInDynamicTemplate()
7570
"full_message", analyzedString(analyzer),
7671
// http://joda-time.sourceforge.net/api-release/org/joda/time/format/DateTimeFormat.html
7772
// http://www.elasticsearch.org/guide/reference/mapping/date-format.html
78-
"timestamp", typeTimeWithMillis(true),
73+
"timestamp", typeTimeWithMillis(),
7974
// to support wildcard searches in source we need to lowercase the content (wildcard search lowercases search term)
80-
"source", analyzedString("analyzer_keyword"));
75+
"source", analyzedString("analyzer_keyword"),
76+
"streams", notAnalyzedString());
77+
}
78+
79+
private Map<String, String> notAnalyzedString() {
80+
return ImmutableMap.of(
81+
"index", "not_analyzed",
82+
"type", "string");
8183
}
8284

8385
private Map<String, String> analyzedString(String analyzer) {
@@ -87,20 +89,13 @@ private Map<String, String> analyzedString(String analyzer) {
8789
"analyzer", analyzer);
8890
}
8991

90-
private Map<String, Serializable> typeTimeWithMillis(boolean storeAsDocValues) {
92+
private Map<String, Serializable> typeTimeWithMillis() {
9193
return ImmutableMap.<String, Serializable>of(
9294
"type", "date",
93-
"format", Tools.ES_DATE_FORMAT,
94-
"doc_values", storeAsDocValues);
95+
"format", Tools.ES_DATE_FORMAT);
9596
}
9697

9798
private Map<String, Boolean> enabled() {
9899
return ImmutableMap.of("enabled", true);
99100
}
100-
101-
private Map<String, Boolean> enabledAndCompressed() {
102-
return ImmutableMap.of(
103-
"enabled", true,
104-
"compress", true);
105-
}
106-
}
101+
}

0 commit comments

Comments
 (0)