Skip to content

Commit

Permalink
Upgrade to Lucene 4.6.1
Browse files Browse the repository at this point in the history 
This upgrade includes a fix for RAM estimation on IndexReader
that allows to expose the amount of used bytes per segment now
as a setting in Elasticsearch. (LUCENE-5373)

Additionally this bugfix release contained a small fix for highlighting
that was already ported to Elasticsearch when reported (LUCENE-5361)

Closes elastic#4897
  • Loading branch information
@s1monw
s1monw committed Jan 28, 2014
1 parent 1da5f3a commit 91acca7
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 123 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
</parent>

<properties>
<lucene.version>4.6.0</lucene.version>
<lucene.version>4.6.1</lucene.version>
<tests.jvms>1</tests.jvms>
<tests.shuffle>true</tests.shuffle>
<tests.output>onerror</tests.output>
Expand Down
40 changes: 3 additions & 37 deletions src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,42 +68,8 @@ public CustomFieldQuery(Query query, IndexReader reader, boolean phraseHighlight

@Override
void flatten(Query sourceQuery, IndexReader reader, Collection<Query> flatQueries) throws IOException {
assert Lucene.VERSION == Version.LUCENE_46 : "LUCENE-5361";
if( sourceQuery instanceof BooleanQuery ){
BooleanQuery bq = (BooleanQuery)sourceQuery;
if (bq.getBoost() == 1) {
for( BooleanClause clause : bq.getClauses() ) {
if(!clause.isProhibited()) {
flatten(clause.getQuery(), reader, flatQueries);
}
}
} else {
for( BooleanClause clause : bq.getClauses() ) {
if(!clause.isProhibited()) {
Query cloned = clause.getQuery().clone();
cloned.setBoost(cloned.getBoost() * bq.getBoost());
flatten(cloned, reader, flatQueries);
}
}
}
} else if (sourceQuery instanceof DisjunctionMaxQuery) {
DisjunctionMaxQuery dmq = (DisjunctionMaxQuery) sourceQuery;
if (dmq.getBoost() == 1) {
for (Query query : dmq) {
flatten(query, reader, flatQueries);
}
} else {
for (Query query : dmq) {
Query clone = query.clone();
clone.setBoost(clone.getBoost() * dmq.getBoost());
flatten(clone, reader, flatQueries);
}
}
} else if (sourceQuery instanceof SpanTermQuery) {
TermQuery termQuery = new TermQuery(((SpanTermQuery) sourceQuery).getTerm());
if (!flatQueries.contains(termQuery)) {
flatQueries.add(termQuery);
}
if (sourceQuery instanceof SpanTermQuery) {
super.flatten(new TermQuery(((SpanTermQuery) sourceQuery).getTerm()), reader, flatQueries);
} else if (sourceQuery instanceof ConstantScoreQuery) {
ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) sourceQuery;
if (constantScoreQuery.getFilter() != null) {
Expand All @@ -128,7 +94,7 @@ void flatten(Query sourceQuery, IndexReader reader, Collection<Query> flatQuerie
convertMultiPhraseQuery(0, new int[q.getTermArrays().size()] , q, q.getTermArrays(), q.getPositions(), reader, flatQueries);
} else {
super.flatten(sourceQuery, reader, flatQueries);
}
}
}

private void convertMultiPhraseQuery(int currentPos, int[] termsIdx, MultiPhraseQuery orig, List<Term[]> terms, int[] pos, IndexReader reader, Collection<Query> flatQueries) throws IOException {
Expand Down
8 changes: 1 addition & 7 deletions src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1119,15 +1119,9 @@ public void recover(RecoveryHandler recoveryHandler) throws EngineException {
}
}

/*this is only used by one test right now and shoudl go away entirely once we update lucene*/
private static boolean allowRamBytesUsed = false;
static {
assert Version.CURRENT.luceneVersion == org.apache.lucene.util.Version.LUCENE_46 :
"when upgrading to a new lucene version, check if ramBytes is fixed, see https://issues.apache.org/jira/browse/LUCENE-5373";
}

private long getReaderRamBytesUsed(AtomicReaderContext reader) {
return allowRamBytesUsed ? SegmentReaderUtils.segmentReader(reader.reader()).ramBytesUsed() : 0;
return SegmentReaderUtils.segmentReader(reader.reader()).ramBytesUsed();
}

@Override
Expand Down
140 changes: 62 additions & 78 deletions src/test/java/org/elasticsearch/index/engine/internal/InternalEngineIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,90 +50,74 @@ public class InternalEngineIntegrationTest extends ElasticsearchIntegrationTest
@Test
@Slow
public void testSettingLoadBloomFilterDefaultTrue() throws Exception {
Field allowRamBytesUsed = InternalEngine.class.getDeclaredField("allowRamBytesUsed");
allowRamBytesUsed.setAccessible(true);
allowRamBytesUsed.set(InternalEngine.class, Boolean.TRUE);

try {
client().admin().indices().prepareCreate("test").setSettings(ImmutableSettings.builder().put("number_of_replicas", 0).put("number_of_shards", 1)).get();
client().prepareIndex("test", "foo").setSource("field", "foo").get();
ensureGreen();
refresh();
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
final long segmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("segments with bloom: {}", segmentsMemoryWithBloom);

logger.info("updating the setting to unload bloom filters");
client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get();
logger.info("waiting for memory to match without blooms");
awaitBusy(new Predicate<Object>() {
public boolean apply(Object o) {
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
long segmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("trying segments without bloom: {}", segmentsMemoryWithoutBloom);
return segmentsMemoryWithoutBloom == (segmentsMemoryWithBloom - BloomFilter.Factory.DEFAULT.createFilter(1).getSizeInBytes());
}
});

logger.info("updating the setting to load bloom filters");
client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, true)).get();
logger.info("waiting for memory to match with blooms");
awaitBusy(new Predicate<Object>() {
public boolean apply(Object o) {
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
long newSegmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("trying segments with bloom: {}", newSegmentsMemoryWithBloom);
return newSegmentsMemoryWithBloom == segmentsMemoryWithBloom;
}
});
} finally {
allowRamBytesUsed.set(InternalEngine.class, Boolean.FALSE);
}
client().admin().indices().prepareCreate("test").setSettings(ImmutableSettings.builder().put("number_of_replicas", 0).put("number_of_shards", 1)).get();
client().prepareIndex("test", "foo").setSource("field", "foo").get();
ensureGreen();
refresh();
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
final long segmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("segments with bloom: {}", segmentsMemoryWithBloom);

logger.info("updating the setting to unload bloom filters");
client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get();
logger.info("waiting for memory to match without blooms");
awaitBusy(new Predicate<Object>() {
public boolean apply(Object o) {
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
long segmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("trying segments without bloom: {}", segmentsMemoryWithoutBloom);
return segmentsMemoryWithoutBloom == (segmentsMemoryWithBloom - BloomFilter.Factory.DEFAULT.createFilter(1).getSizeInBytes());
}
});

logger.info("updating the setting to load bloom filters");
client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, true)).get();
logger.info("waiting for memory to match with blooms");
awaitBusy(new Predicate<Object>() {
public boolean apply(Object o) {
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
long newSegmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("trying segments with bloom: {}", newSegmentsMemoryWithBloom);
return newSegmentsMemoryWithBloom == segmentsMemoryWithBloom;
}
});
}

@Test
@Slow
public void testSettingLoadBloomFilterDefaultFalse() throws Exception {
Field allowRamBytesUsed = InternalEngine.class.getDeclaredField("allowRamBytesUsed");
allowRamBytesUsed.setAccessible(true);
allowRamBytesUsed.set(InternalEngine.class, Boolean.TRUE);
try {
client().admin().indices().prepareCreate("test").setSettings(ImmutableSettings.builder().put("number_of_replicas", 0).put("number_of_shards", 1).put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get();
client().prepareIndex("test", "foo").setSource("field", "foo").get();
ensureGreen();
refresh();

IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
final long segmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("segments without bloom: {}", segmentsMemoryWithoutBloom);

logger.info("updating the setting to load bloom filters");
client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, true)).get();
logger.info("waiting for memory to match with blooms");
awaitBusy(new Predicate<Object>() {
public boolean apply(Object o) {
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
long segmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("trying segments with bloom: {}", segmentsMemoryWithoutBloom);
return segmentsMemoryWithoutBloom == (segmentsMemoryWithBloom - BloomFilter.Factory.DEFAULT.createFilter(1).getSizeInBytes());
}
});

logger.info("updating the setting to unload bloom filters");
client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get();
logger.info("waiting for memory to match without blooms");
awaitBusy(new Predicate<Object>() {
public boolean apply(Object o) {
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
long newSegmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("trying segments without bloom: {}", newSegmentsMemoryWithoutBloom);
return newSegmentsMemoryWithoutBloom == segmentsMemoryWithoutBloom;
}
});
} finally {
allowRamBytesUsed.set(InternalEngine.class, Boolean.FALSE);
}
client().admin().indices().prepareCreate("test").setSettings(ImmutableSettings.builder().put("number_of_replicas", 0).put("number_of_shards", 1).put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get();
client().prepareIndex("test", "foo").setSource("field", "foo").get();
ensureGreen();
refresh();

IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
final long segmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("segments without bloom: {}", segmentsMemoryWithoutBloom);

logger.info("updating the setting to load bloom filters");
client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, true)).get();
logger.info("waiting for memory to match with blooms");
awaitBusy(new Predicate<Object>() {
public boolean apply(Object o) {
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
long segmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("trying segments with bloom: {}", segmentsMemoryWithoutBloom);
return segmentsMemoryWithoutBloom == (segmentsMemoryWithBloom - BloomFilter.Factory.DEFAULT.createFilter(1).getSizeInBytes());
}
});

logger.info("updating the setting to unload bloom filters");
client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get();
logger.info("waiting for memory to match without blooms");
awaitBusy(new Predicate<Object>() {
public boolean apply(Object o) {
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
long newSegmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes();
logger.info("trying segments without bloom: {}", newSegmentsMemoryWithoutBloom);
return newSegmentsMemoryWithoutBloom == segmentsMemoryWithoutBloom;
}
});
}

@Test
Expand Down

0 comments on commit 91acca7

Please sign in to comment.