Skip to content

Commit 2d7b921

Browse files
joschibernd
joschi
authored and
bernd
committed
Upgrade to Elasticsearch 5.6 (Graylog2#4249)
* Upgrade to Elasticsearch 5.6.4 Upgrade the internal ES dependency to Elasticsearch 5.6.4 and adapt our code to API changes. * Remove Elasticsearch 2.x support * Fix highlighting
1 parent 3f930e0 commit 2d7b921

File tree

13 files changed

+35
-244
lines changed

13 files changed

+35
-244
lines changed

‎.travis.yml

-1
Original file line numberDiff line numberDiff line change
@@ -51,5 +51,4 @@ env:
5151
- secure: r0enVlGBPjqaGfNrD1b6YxgRaSp/v39ZMKQFs/C0RtRKliZ5LVo7kS4lL0tLQoqMAXJel0NgsVH6j1P6428uLQDxqKmCsNZiX0VS9K3Z0iq5NG4x/5QidxyXjHefSGmpquLmZbM094xhtvdou3NEfkJTKbxHu3ML9i9ScMn5+vA=
5252
- secure: "OiqpeUFVXO/sWdSA4WZIrK5UvVUN4lG5YWm41e1P+6lvOAHpWeyX6fIs9Ndz12Vio/Wlt03ixzCAO/67UW4XGv/e3tClmvOSKt811Ml8wu3ALMxdV2218OkFXGzQ8AOnIRK1AAln3JdvBMHj8wFoUmGBaHJ7odqOxu3WC4m0Xr0="
5353
matrix:
54-
- GRAYLOG_ELASTICSEARCH_VERSION=2
5554
- GRAYLOG_ELASTICSEARCH_VERSION=5

‎graylog-project-parent/pom.xml

+1-7
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,6 @@
117117
</exclusions>
118118
</dependency>
119119

120-
<dependency>
121-
<groupId>net.java.dev.jna</groupId>
122-
<artifactId>jna</artifactId>
123-
<version>${jna.version}</version>
124-
</dependency>
125-
126120
<dependency>
127121
<groupId>org.kie</groupId>
128122
<artifactId>kie-api</artifactId>
@@ -541,7 +535,7 @@
541535
<dependency>
542536
<groupId>com.github.joschi.nosqlunit</groupId>
543537
<artifactId>nosqlunit-elasticsearch-http</artifactId>
544-
<version>1.0.0-4+jackson</version>
538+
<version>${nosqlunit-elasticsearch-http.version}</version>
545539
<scope>test</scope>
546540
</dependency>
547541
<dependency>

‎graylog2-server/pom.xml

+2-88
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565

6666
<it.es.httpPort>19200</it.es.httpPort>
6767
<it.es.transportPort>19300</it.es.transportPort>
68-
<it.es.version>5.5.2</it.es.version>
68+
<it.es.version>5.6.4</it.es.version>
6969
<it.es.instanceCount>1</it.es.instanceCount>
7070
<it.es.clusterName>test</it.es.clusterName>
7171
<it.es.await>false</it.es.await>
@@ -274,11 +274,6 @@
274274
<artifactId>jest</artifactId>
275275
</dependency>
276276

277-
<dependency>
278-
<groupId>net.java.dev.jna</groupId>
279-
<artifactId>jna</artifactId>
280-
</dependency>
281-
282277
<dependency>
283278
<groupId>org.kie</groupId>
284279
<artifactId>kie-api</artifactId>
@@ -356,10 +351,6 @@
356351
<groupId>org.assertj</groupId>
357352
<artifactId>assertj-joda-time</artifactId>
358353
</dependency>
359-
<dependency>
360-
<groupId>com.lordofthejars</groupId>
361-
<artifactId>nosqlunit-elasticsearch2</artifactId>
362-
</dependency>
363354
<dependency>
364355
<groupId>com.lordofthejars</groupId>
365356
<artifactId>nosqlunit-mongodb</artifactId>
@@ -772,83 +763,6 @@
772763
</plugins>
773764
</build>
774765
</profile>
775-
<profile>
776-
<id>integration-tests-elasticsearch-2</id>
777-
<activation>
778-
<property>
779-
<name>it.ElasticsearchVersion</name>
780-
<value>2</value>
781-
</property>
782-
</activation>
783-
<properties>
784-
<it.es.version>2.4.6</it.es.version>
785-
</properties>
786-
<build>
787-
<plugins>
788-
<plugin>
789-
<groupId>org.apache.maven.plugins</groupId>
790-
<artifactId>maven-failsafe-plugin</artifactId>
791-
<configuration>
792-
<forkCount>1</forkCount>
793-
<skip>${it.es.skip}</skip>
794-
</configuration>
795-
<executions>
796-
<execution>
797-
<goals>
798-
<goal>integration-test</goal>
799-
<goal>verify</goal>
800-
</goals>
801-
</execution>
802-
</executions>
803-
</plugin>
804-
<plugin>
805-
<groupId>com.github.alexcojocaru</groupId>
806-
<artifactId>elasticsearch-maven-plugin</artifactId>
807-
<version>2.3</version>
808-
<configuration>
809-
<skip>${it.es.skip}</skip>
810-
<clusterName>${it.es.clusterName}</clusterName>
811-
<httpPort>${it.es.httpPort}</httpPort>
812-
<tcpPort>${it.es.transportPort}</tcpPort>
813-
<autoCreateIndex>true</autoCreateIndex>
814-
<configPath>${project.basedir}/src/test/resources/it/elasticsearch2</configPath>
815-
<logsDirname>${project.build.directory}/elasticsearch2-logs</logsDirname>
816-
<scriptFile/>
817-
</configuration>
818-
<executions>
819-
<!-- The elasticsearch plugin is by default bound to the
820-
pre-integration-test and post-integration-test phases -->
821-
<execution>
822-
<id>start-elasticsearch</id>
823-
<phase>pre-integration-test</phase>
824-
<goals>
825-
<goal>start</goal>
826-
</goals>
827-
</execution>
828-
<execution>
829-
<id>stop-elasticsearch</id>
830-
<phase>post-integration-test</phase>
831-
<goals>
832-
<goal>stop</goal>
833-
</goals>
834-
</execution>
835-
</executions>
836-
<dependencies>
837-
<dependency>
838-
<groupId>org.elasticsearch</groupId>
839-
<artifactId>elasticsearch</artifactId>
840-
<version>${it.es.version}</version>
841-
</dependency>
842-
<dependency>
843-
<groupId>log4j</groupId>
844-
<artifactId>log4j</artifactId>
845-
<version>1.2.17</version>
846-
</dependency>
847-
</dependencies>
848-
</plugin>
849-
</plugins>
850-
</build>
851-
</profile>
852766
<profile>
853767
<id>integration-tests-elasticsearch-5</id>
854768
<activation>
@@ -859,7 +773,7 @@
859773
</property>
860774
</activation>
861775
<properties>
862-
<it.es.version>5.5.2</it.es.version>
776+
<it.es.version>5.6.4</it.es.version>
863777
</properties>
864778
<build>
865779
<plugins>

‎graylog2-server/src/main/java/org/graylog2/indexer/IndexMapping2.java

-56
This file was deleted.

‎graylog2-server/src/main/java/org/graylog2/indexer/IndexMappingFactory.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,7 @@ public IndexMappingFactory(Node node) {
3333

3434
public IndexMapping createIndexMapping() {
3535
final Version elasticsearchVersion = node.getVersion().orElseThrow(() -> new ElasticsearchException("Unable to retrieve Elasticsearch version."));
36-
if (elasticsearchVersion.satisfies(">=2.1.0 & <5.0.0")) {
37-
return new IndexMapping2();
38-
} else if (elasticsearchVersion.satisfies("^5.0.0")) {
36+
if (elasticsearchVersion.satisfies("^5.0.0")) {
3937
return new IndexMapping5();
4038
} else {
4139
throw new ElasticsearchException("Unsupported Elasticsearch version: " + elasticsearchVersion);

‎graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@
6565
import org.elasticsearch.search.aggregations.AggregationBuilders;
6666
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
6767
import org.elasticsearch.search.builder.SearchSourceBuilder;
68+
import org.elasticsearch.search.sort.FieldSortBuilder;
6869
import org.elasticsearch.search.sort.SortBuilders;
69-
import org.elasticsearch.search.sort.SortParseElement;
7070
import org.graylog2.audit.AuditActor;
7171
import org.graylog2.audit.AuditEventSender;
7272
import org.graylog2.indexer.ElasticsearchException;
@@ -141,7 +141,7 @@ public void move(String source, String target) {
141141
final String query = SearchSourceBuilder.searchSource()
142142
.query(QueryBuilders.matchAllQuery())
143143
.size(350)
144-
.sort(SortBuilders.fieldSort(SortParseElement.DOC_FIELD_NAME))
144+
.sort(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME))
145145
.toString();
146146

147147
final Search request = new Search.Builder(query)
@@ -666,8 +666,7 @@ public Optional<DateTime> indexCreationDate(String index) {
666666
* @see org.elasticsearch.search.aggregations.metrics.stats.Stats
667667
*/
668668
public IndexRangeStats indexRangeStatsOfIndex(String index) {
669-
final FilterAggregationBuilder builder = AggregationBuilders.filter("agg")
670-
.filter(QueryBuilders.existsQuery(Message.FIELD_TIMESTAMP))
669+
final FilterAggregationBuilder builder = AggregationBuilders.filter("agg", QueryBuilders.existsQuery(Message.FIELD_TIMESTAMP))
671670
.subAggregation(AggregationBuilders.min("ts_min").field(Message.FIELD_TIMESTAMP))
672671
.subAggregation(AggregationBuilders.max("ts_max").field(Message.FIELD_TIMESTAMP))
673672
.subAggregation(AggregationBuilders.terms("streams").field(Message.FIELD_STREAMS));

‎graylog2-server/src/main/java/org/graylog2/indexer/results/FieldHistogramResult.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ private void fillEmptyTimestamps(Map<Long, Map<String, Number>> results) {
107107
while (currentTime.getMillis() < maxTimestamp) {
108108
final Map<String, Number> entry = results.get(currentTime.getMillis());
109109

110-
// advance timestamp by the interval's seconds value
110+
// advance timestamp by the interval
111111
currentTime.add(interval.getPeriod());
112112

113113
if (entry == null) {

‎graylog2-server/src/main/java/org/graylog2/indexer/searches/Searches.java

+17-20
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,14 @@
4343
import org.elasticsearch.index.query.QueryBuilder;
4444
import org.elasticsearch.index.query.QueryBuilders;
4545
import org.elasticsearch.script.Script;
46-
import org.elasticsearch.script.ScriptService;
46+
import org.elasticsearch.script.ScriptType;
4747
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
4848
import org.elasticsearch.search.aggregations.AggregationBuilders;
4949
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
50-
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramBuilder;
50+
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
5151
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
5252
import org.elasticsearch.search.builder.SearchSourceBuilder;
53+
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
5354
import org.graylog2.Configuration;
5455
import org.graylog2.database.NotFoundException;
5556
import org.graylog2.indexer.ElasticsearchException;
@@ -296,8 +297,7 @@ public AbstractAggregationBuilder createTermsBuilder(String field, List<String>
296297
if (stackedFields.isEmpty()) {
297298
// Wrap terms aggregation in a no-op filter to make sure the result structure is correct when not having
298299
// stacked fields.
299-
return AggregationBuilders.filter(AGG_FILTER)
300-
.filter(QueryBuilders.matchAllQuery())
300+
return AggregationBuilders.filter(AGG_FILTER, QueryBuilders.matchAllQuery())
301301
.subAggregation(AggregationBuilders.terms(AGG_TERMS)
302302
.field(field)
303303
.size(size > 0 ? size : 50)
@@ -326,10 +326,9 @@ public AbstractAggregationBuilder createTermsBuilder(String field, List<String>
326326
filterQuery.must(QueryBuilders.existsQuery(f));
327327
});
328328

329-
return AggregationBuilders.filter(AGG_FILTER)
330-
.filter(filterQuery)
329+
return AggregationBuilders.filter(AGG_FILTER, filterQuery)
331330
.subAggregation(AggregationBuilders.terms(AGG_TERMS)
332-
.script(new Script(scriptStringBuilder.toString(), ScriptService.ScriptType.INLINE, "painless", null))
331+
.script(new Script(ScriptType.INLINE, "painless", scriptStringBuilder.toString(), Collections.emptyMap()))
333332
.size(size > 0 ? size : 50)
334333
.order(termsOrder));
335334
}
@@ -393,9 +392,9 @@ public TermsHistogramResult termsHistogram(String field,
393392
Sorting.Direction sorting) {
394393
final Terms.Order termsOrder = sorting == Sorting.Direction.DESC ? Terms.Order.count(false) : Terms.Order.count(true);
395394

396-
final DateHistogramBuilder histogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
395+
final DateHistogramAggregationBuilder histogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
397396
.field(Message.FIELD_TIMESTAMP)
398-
.interval(interval.toESInterval())
397+
.dateHistogramInterval(interval.toESInterval())
399398
.subAggregation(createTermsBuilder(field, stackedFields, size, termsOrder))
400399
.subAggregation(AggregationBuilders.missing("missing").field(field));
401400

@@ -484,16 +483,14 @@ public TermsStatsResult termsStats(String keyField, String valueField, TermsStat
484483
termsOrder = Terms.Order.count(true);
485484
}
486485

487-
final FilterAggregationBuilder builder = AggregationBuilders.filter(AGG_FILTER)
486+
final FilterAggregationBuilder builder = AggregationBuilders.filter(AGG_FILTER, standardAggregationFilters(range, filter))
488487
.subAggregation(
489488
AggregationBuilders.terms(AGG_TERMS_STATS)
490489
.field(keyField)
491490
.subAggregation(AggregationBuilders.stats(AGG_STATS).field(valueField))
492491
.order(termsOrder)
493492
.size(size)
494-
)
495-
.filter(standardAggregationFilters(range, filter));
496-
493+
);
497494
searchSourceBuilder.aggregation(builder);
498495

499496
if (affectedIndices.isEmpty()) {
@@ -546,8 +543,7 @@ public FieldStatsResult fieldStats(String field,
546543
searchSourceBuilder = filteredSearchRequest(query, filter, range);
547544
}
548545

549-
final FilterAggregationBuilder filterBuilder = AggregationBuilders.filter(AGG_FILTER)
550-
.filter(standardAggregationFilters(range, filter));
546+
final FilterAggregationBuilder filterBuilder = AggregationBuilders.filter(AGG_FILTER, standardAggregationFilters(range, filter));
551547
if (includeCount) {
552548
searchSourceBuilder.aggregation(AggregationBuilders.count(AGG_VALUE_COUNT).field(field));
553549
}
@@ -605,9 +601,9 @@ public HistogramResult histogram(String query, DateHistogramInterval interval, T
605601
}
606602

607603
public HistogramResult histogram(String query, DateHistogramInterval interval, String filter, TimeRange range) {
608-
final DateHistogramBuilder histogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
604+
final DateHistogramAggregationBuilder histogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
609605
.field(Message.FIELD_TIMESTAMP)
610-
.interval(interval.toESInterval());
606+
.dateHistogramInterval(interval.toESInterval());
611607

612608
final SearchSourceBuilder searchSourceBuilder = filteredSearchRequest(query, filter, range)
613609
.aggregation(histogramBuilder);
@@ -653,9 +649,9 @@ public HistogramResult fieldHistogram(String query,
653649
TimeRange range,
654650
boolean includeStats,
655651
boolean includeCardinality) {
656-
final DateHistogramBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
652+
final DateHistogramAggregationBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram(AGG_HISTOGRAM)
657653
.field(Message.FIELD_TIMESTAMP)
658-
.interval(interval.toESInterval());
654+
.dateHistogramInterval(interval.toESInterval());
659655

660656
if (includeStats) {
661657
dateHistogramBuilder.subAggregation(AggregationBuilders.stats(AGG_STATS).field(field));
@@ -797,11 +793,12 @@ private SearchSourceBuilder standardSearchRequest(
797793
}
798794

799795
if (highlight && configuration.isAllowHighlighting()) {
800-
searchSourceBuilder.highlighter()
796+
final HighlightBuilder highlightBuilder = new HighlightBuilder()
801797
.requireFieldMatch(false)
802798
.field("*")
803799
.fragmentSize(0)
804800
.numOfFragments(0);
801+
searchSourceBuilder.highlighter(highlightBuilder);
805802
}
806803

807804
return searchSourceBuilder;

‎graylog2-server/src/test/java/org/graylog2/ElasticsearchBase.java

+1-4
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import io.searchbox.indices.template.GetTemplate;
4040
import io.searchbox.indices.template.PutTemplate;
4141
import org.graylog2.indexer.IndexMapping;
42-
import org.graylog2.indexer.IndexMapping2;
4342
import org.graylog2.indexer.IndexMapping5;
4443
import org.junit.Rule;
4544

@@ -120,12 +119,10 @@ protected void assertSucceeded(JestResult jestResult) {
120119

121120
protected IndexMapping indexMapping() {
122121
switch (elasticsearchVersion.getMajorVersion()) {
123-
case 2:
124-
return new IndexMapping2();
125122
case 5:
126123
return new IndexMapping5();
127124
default:
128-
throw new IllegalStateException("Only Elasticsearch 2.x and 5.x are supported");
125+
throw new IllegalStateException("Only Elasticsearch 5.x is supported");
129126
}
130127
}
131128

0 commit comments

Comments
 (0)