Skip to content

Commit

Permalink
FieldData Stats: Add field data stats to indices stats API
Browse files Browse the repository at this point in the history 
closes elastic#2870
  • Loading branch information
@kimchy
kimchy committed Apr 8, 2013
1 parent 86c1714 commit 15d7ae5
Show file tree
Hide file tree
Showing 17 changed files with 392 additions and 84 deletions.
31 changes: 30 additions & 1 deletion src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.cache.filter.FilterCacheStats;
import org.elasticsearch.index.cache.id.IdCacheStats;
import org.elasticsearch.index.fielddata.FieldDataStats;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.indexing.IndexingStats;
Expand Down Expand Up @@ -76,6 +77,9 @@ public class CommonStats implements Streamable, ToXContent {
@Nullable
IdCacheStats idCache;

@Nullable
FieldDataStats fieldData;

public void add(CommonStats stats) {
if (docs == null) {
if (stats.getDocs() != null) {
Expand Down Expand Up @@ -166,6 +170,15 @@ public void add(CommonStats stats) {
} else {
idCache.add(stats.getIdCache());
}

if (fieldData == null) {
if (stats.getFieldData() != null) {
fieldData = new FieldDataStats();
fieldData.add(stats.getFieldData());
}
} else {
fieldData.add(stats.getFieldData());
}
}

@Nullable
Expand Down Expand Up @@ -223,6 +236,11 @@ public IdCacheStats getIdCache() {
return this.idCache;
}

@Nullable
public FieldDataStats getFieldData() {
return this.fieldData;
}

public static CommonStats readCommonStats(StreamInput in) throws IOException {
CommonStats stats = new CommonStats();
stats.readFrom(in);
Expand Down Expand Up @@ -264,6 +282,9 @@ public void readFrom(StreamInput in) throws IOException {
if (in.readBoolean()) {
idCache = IdCacheStats.readIdCacheStats(in);
}
if (in.readBoolean()) {
fieldData = FieldDataStats.readFieldDataStats(in);
}
}

@Override
Expand Down Expand Up @@ -328,13 +349,18 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(true);
filterCache.writeTo(out);
}

if (idCache == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
idCache.writeTo(out);
}
if (fieldData == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
fieldData.writeTo(out);
}
}

// note, requires a wrapping object
Expand Down Expand Up @@ -373,6 +399,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (idCache != null) {
idCache.toXContent(builder, params);
}
if (fieldData != null) {
fieldData.toXContent(builder, params);
}
return builder;
}
}
14 changes: 14 additions & 0 deletions src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest<IndicesStatsR
private boolean warmer = false;
private boolean filterCache = false;
private boolean idCache = false;
private boolean fieldData = false;
private String[] types = null;
private String[] groups = null;

Expand All @@ -65,6 +66,7 @@ public IndicesStatsRequest all() {
warmer = true;
filterCache = true;
idCache = true;
fieldData = true;
types = null;
groups = null;
return this;
Expand All @@ -85,6 +87,7 @@ public IndicesStatsRequest clear() {
warmer = false;
filterCache = false;
idCache = false;
fieldData = false;
types = null;
groups = null;
return this;
Expand Down Expand Up @@ -219,6 +222,15 @@ public boolean idCache() {
return this.idCache;
}

public IndicesStatsRequest fieldData(boolean fieldData) {
this.fieldData = fieldData;
return this;
}

public boolean fieldData() {
return this.fieldData;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
Expand All @@ -233,6 +245,7 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(warmer);
out.writeBoolean(filterCache);
out.writeBoolean(idCache);
out.writeBoolean(fieldData);
if (types == null) {
out.writeVInt(0);
} else {
Expand Down Expand Up @@ -265,6 +278,7 @@ public void readFrom(StreamInput in) throws IOException {
warmer = in.readBoolean();
filterCache = in.readBoolean();
idCache = in.readBoolean();
fieldData = in.readBoolean();
int size = in.readVInt();
if (size > 0) {
types = new String[size];
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,11 @@ public IndicesStatsRequestBuilder setIdCache(boolean idCache) {
return this;
}

public IndicesStatsRequestBuilder setFieldData(boolean fieldData) {
request.fieldData(fieldData);
return this;
}

@Override
protected void doExecute(ActionListener<IndicesStatsResponse> listener) {
((IndicesAdminClient) client).stats(request, listener);
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,9 @@ protected ShardStats shardOperation(IndexShardStatsRequest request) throws Elast
if (request.request.idCache()) {
stats.stats.idCache = indexShard.idCacheStats();
}
if (request.request.fieldData()) {
stats.stats.fieldData = indexShard.fieldDataStats();
}

return stats;
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ public FieldMapper.Names getFieldNames() {

@Override
public void clear() {
cache.clear(index, fieldNames.indexName());
cache.clear(fieldNames.indexName());
}

@Override
public void clear(IndexReader reader) {
cache.clear(index, reader);
cache.clear(reader);
}
}
113 changes: 82 additions & 31 deletions src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,103 +27,154 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SegmentReader;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardUtils;
import org.elasticsearch.index.shard.service.IndexShard;

import java.util.concurrent.Callable;

/**
* A simple field data cache abstraction.
* A simple field data cache abstraction on the *index* level.
*/
public interface IndexFieldDataCache {

<FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(AtomicReaderContext context, IFD indexFieldData) throws Exception;

void clear(Index index);
/**
* Clears all the field data stored cached in on this index.
*/
void clear();

void clear(Index index, String fieldName);
/**
* Clears all the field data stored cached in on this index for the specified field name.
*/
void clear(String fieldName);

void clear(Index index, IndexReader reader);
void clear(IndexReader reader);

interface Listener {

void onLoad(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData);
void onLoad(FieldMapper.Names fieldNames, FieldDataType fieldDataType, AtomicFieldData fieldData);

void onUnload(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, @Nullable AtomicFieldData fieldData);
void onUnload(FieldMapper.Names fieldNames, FieldDataType fieldDataType, boolean wasEvicted, long sizeInBytes, @Nullable AtomicFieldData fieldData);
}

/**
* The resident field data cache is a *per field* cache that keeps all the values in memory.
*/
static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener, RemovalListener<Object, AtomicFieldData> {
private final Index index;
static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener, RemovalListener<FieldBased.Key, AtomicFieldData> {
@Nullable
private final IndexService indexService;
private final FieldMapper.Names fieldNames;
private final FieldDataType fieldDataType;
private final Listener listener;
private final Cache<Object, AtomicFieldData> cache;
private final Cache<Key, AtomicFieldData> cache;

protected FieldBased(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener, CacheBuilder cache) {
this.index = index;
protected FieldBased(@Nullable IndexService indexService, FieldMapper.Names fieldNames, FieldDataType fieldDataType, CacheBuilder cache) {
this.indexService = indexService;
this.fieldNames = fieldNames;
this.fieldDataType = fieldDataType;
this.listener = listener;
cache.removalListener(this);
this.cache = cache.build();
}

@Override
public void onRemoval(RemovalNotification<Object, AtomicFieldData> notification) {
listener.onUnload(index, fieldNames, fieldDataType, notification.wasEvicted(), notification.getValue());
}

@Override
public void onClose(SegmentReader owner) {
cache.invalidate(owner.getCoreCacheKey());
public void onRemoval(RemovalNotification<Key, AtomicFieldData> notification) {
if (notification.getKey() != null) {
notification.getKey().listener.onUnload(fieldNames, fieldDataType, notification.wasEvicted(), notification.getKey().sizeInBytes, notification.getValue());
}
}

@Override
public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final AtomicReaderContext context, final IFD indexFieldData) throws Exception {
final Key key = new Key(context.reader().getCoreCacheKey());
//noinspection unchecked
return (FD) cache.get(context.reader().getCoreCacheKey(), new Callable<AtomicFieldData>() {
return (FD) cache.get(key, new Callable<AtomicFieldData>() {
@Override
public AtomicFieldData call() throws Exception {
if (context.reader() instanceof SegmentReader) {
((SegmentReader) context.reader()).addCoreClosedListener(FieldBased.this);
}

AtomicFieldData fieldData = indexFieldData.loadDirect(context);
listener.onLoad(index, fieldNames, fieldDataType, fieldData);
key.sizeInBytes = fieldData.getMemorySizeInBytes();

if (indexService != null) {
ShardId shardId = ShardUtils.extractShardId(context.reader());
if (shardId != null) {
IndexShard shard = indexService.shard(shardId.id());
if (shard != null) {
key.listener = shard.fieldData();
}
}
}

if (key.listener != null) {
key.listener.onLoad(fieldNames, fieldDataType, fieldData);
}

return fieldData;
}
});
}

@Override
public void clear(Index index) {
public void clear() {
cache.invalidateAll();
}

@Override
public void clear(Index index, String fieldName) {
public void clear(String fieldName) {
cache.invalidateAll();
}

@Override
public void clear(Index index, IndexReader reader) {
cache.invalidate(reader.getCoreCacheKey());
public void clear(IndexReader reader) {
cache.invalidate(new Key(reader.getCoreCacheKey()));
}

@Override
public void onClose(SegmentReader owner) {
cache.invalidate(new Key(owner.getCoreCacheKey()));
}

static class Key {
final Object readerKey;
@Nullable
Listener listener; // optional stats listener
long sizeInBytes = -1; // optional size in bytes (we keep it here in case the values are soft references)

Key(Object readerKey) {
this.readerKey = readerKey;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Key key = (Key) o;
if (!readerKey.equals(key.readerKey)) return false;
return true;
}

@Override
public int hashCode() {
return readerKey.hashCode();
}
}
}

static class Resident extends FieldBased {

public Resident(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder());
public Resident(@Nullable IndexService indexService, FieldMapper.Names fieldNames, FieldDataType fieldDataType) {
super(indexService, fieldNames, fieldDataType, CacheBuilder.newBuilder());
}
}

static class Soft extends FieldBased {

public Soft(Index index, FieldMapper.Names fieldNames, FieldDataType fieldDataType, Listener listener) {
super(index, fieldNames, fieldDataType, listener, CacheBuilder.newBuilder().softValues());
public Soft(@Nullable IndexService indexService, FieldMapper.Names fieldNames, FieldDataType fieldDataType) {
super(indexService, fieldNames, fieldDataType, CacheBuilder.newBuilder().softValues());
}
}
}
Loading

0 comments on commit 15d7ae5

Please sign in to comment.