Skip to content

Commit

Permalink
Binary Mapped Fields: Allow to not store them by default, and return …
Browse files Browse the repository at this point in the history 
…BytesReference

fixes elastic#2523
also, fix another point of normalization of the result for get API
  • Loading branch information
@kimchy
kimchy committed Jan 5, 2013
1 parent 4b9fcdb commit 0e5287f
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 15 deletions.
6 changes: 5 additions & 1 deletion src/main/java/org/elasticsearch/index/get/ShardGetService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,10 @@ public GetResult innerGet(String type, String id, String[] gFields, boolean real
// only if the field is stored or source is enabled we should add it..
if (docMapper.sourceMapper().enabled() || x == null || x.fieldType().stored()) {
value = searchLookup.source().extractValue(field);
// normalize the data if needed (mainly for binary fields, to convert from base64 strings to bytes)
if (value != null && x != null) {
value = x.valueForSearch(value);
}
}
}
}
Expand Down Expand Up @@ -326,7 +330,7 @@ private GetResult innerGetLoadFromStoredFields(String type, String id, String[]
}
value = searchLookup.source().extractValue(field);
// normalize the data if needed (mainly for binary fields, to convert from base64 strings to bytes)
if (value != null) {
if (value != null && x != null) {
value = x.mapper().valueForSearch(value);
}
}
Expand Down
12 changes: 8 additions & 4 deletions src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
/**
*
*/
public class BinaryFieldMapper extends AbstractFieldMapper<byte[]> {
public class BinaryFieldMapper extends AbstractFieldMapper<BytesReference> {

public static final String CONTENT_TYPE = "binary";

Expand All @@ -55,7 +55,8 @@ public static class Defaults extends AbstractFieldMapper.Defaults {
public static final FieldType FIELD_TYPE = new FieldType(AbstractFieldMapper.Defaults.FIELD_TYPE);

static {
FIELD_TYPE.setStored(false);
FIELD_TYPE.setIndexed(false);
FIELD_TYPE.setStored(true);
FIELD_TYPE.freeze();
}
}
Expand Down Expand Up @@ -137,7 +138,7 @@ public Object valueForSearch(Object value) {
}

@Override
public byte[] value(Object value) {
public BytesReference value(Object value) {
if (value == null) {
return null;
}
Expand All @@ -157,7 +158,7 @@ public byte[] value(Object value) {
}
}
try {
return CompressorFactory.uncompressIfNeeded(bytes).toBytes();
return CompressorFactory.uncompressIfNeeded(bytes);
} catch (IOException e) {
throw new ElasticSearchParseException("failed to decompress source", e);
}
Expand Down Expand Up @@ -210,6 +211,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (compressThreshold != -1) {
builder.field("compress_threshold", new ByteSizeValue(compressThreshold).toString());
}
if (fieldType.stored() != defaultFieldType().stored()) {
builder.field("store", fieldType.stored());
}
builder.endObject();
return builder;
}
Expand Down
25 changes: 16 additions & 9 deletions src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,10 @@
import org.elasticsearch.action.get.MultiGetRequest;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.test.integration.AbstractNodesTests;
Expand All @@ -39,8 +42,7 @@
import static org.elasticsearch.client.Requests.clusterHealthRequest;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.*;

public class GetActionTests extends AbstractNodesTests {

Expand Down Expand Up @@ -238,6 +240,7 @@ public void getFieldsWithDifferentTypes() throws Exception {
.startObject("str").field("type", "string").field("store", "yes").endObject()
.startObject("int").field("type", "integer").field("store", "yes").endObject()
.startObject("date").field("type", "date").field("store", "yes").endObject()
.startObject("binary").field("type", "binary").field("store", "yes").endObject()
.endObject()
.endObject().endObject())
.execute().actionGet();
Expand All @@ -246,40 +249,44 @@ public void getFieldsWithDifferentTypes() throws Exception {
assertThat(clusterHealth.timedOut(), equalTo(false));
assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN));

client.prepareIndex("test", "type1", "1").setSource("str", "test", "int", 42, "date", "2012-11-13T15:26:14.000Z").execute().actionGet();
client.prepareIndex("test", "type2", "1").setSource("str", "test", "int", 42, "date", "2012-11-13T15:26:14.000Z").execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource("str", "test", "int", 42, "date", "2012-11-13T15:26:14.000Z", "binary", Base64.encodeBytes(new byte[]{1, 2, 3})).execute().actionGet();
client.prepareIndex("test", "type2", "1").setSource("str", "test", "int", 42, "date", "2012-11-13T15:26:14.000Z", "binary", Base64.encodeBytes(new byte[]{1, 2, 3})).execute().actionGet();

// realtime get with stored source
logger.info("--> realtime get (from source)");
GetResponse getResponse = client.prepareGet("test", "type1", "1").setFields("str", "int", "date").execute().actionGet();
GetResponse getResponse = client.prepareGet("test", "type1", "1").setFields("str", "int", "date", "binary").execute().actionGet();
assertThat(getResponse.exists(), equalTo(true));
assertThat((String) getResponse.field("str").getValue(), equalTo("test"));
assertThat((Integer) getResponse.field("int").getValue(), equalTo(42));
assertThat((Long) getResponse.field("int").getValue(), equalTo(42l));
assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z"));
assertThat(getResponse.field("binary").getValue(), instanceOf(String.class)); // its a String..., not binary mapped

logger.info("--> realtime get (from stored fields)");
getResponse = client.prepareGet("test", "type2", "1").setFields("str", "int", "date").execute().actionGet();
getResponse = client.prepareGet("test", "type2", "1").setFields("str", "int", "date", "binary").execute().actionGet();
assertThat(getResponse.exists(), equalTo(true));
assertThat((String) getResponse.field("str").getValue(), equalTo("test"));
assertThat((Integer) getResponse.field("int").getValue(), equalTo(42));
assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z"));
assertThat((BytesReference) getResponse.field("binary").getValue(), equalTo((BytesReference) new BytesArray(new byte[]{1, 2, 3})));

logger.info("--> flush the index, so we load it from it");
client.admin().indices().prepareFlush().execute().actionGet();

logger.info("--> non realtime get (from source)");
getResponse = client.prepareGet("test", "type1", "1").setFields("str", "int", "date").execute().actionGet();
getResponse = client.prepareGet("test", "type1", "1").setFields("str", "int", "date", "binary").execute().actionGet();
assertThat(getResponse.exists(), equalTo(true));
assertThat((String) getResponse.field("str").getValue(), equalTo("test"));
assertThat((Long) getResponse.field("int").getValue(), equalTo(42l));
assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z"));
assertThat(getResponse.field("binary").getValue(), instanceOf(String.class)); // its a String..., not binary mapped

logger.info("--> non realtime get (from stored fields)");
getResponse = client.prepareGet("test", "type2", "1").setFields("str", "int", "date").execute().actionGet();
getResponse = client.prepareGet("test", "type2", "1").setFields("str", "int", "date", "binary").execute().actionGet();
assertThat(getResponse.exists(), equalTo(true));
assertThat((String) getResponse.field("str").getValue(), equalTo("test"));
assertThat((Integer) getResponse.field("int").getValue(), equalTo(42));
assertThat((String) getResponse.field("date").getValue(), equalTo("2012-11-13T15:26:14.000Z"));
assertThat((BytesReference) getResponse.field("binary").getValue(), equalTo((BytesReference) new BytesArray(new byte[]{1, 2, 3})));
}

@Test
Expand Down
4 changes: 3 additions & 1 deletion src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.xcontent.XContentFactory;
Expand Down Expand Up @@ -343,7 +345,7 @@ public void testStoredFieldsWithoutSource() throws Exception {
String dateTime = Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC));
assertThat(searchResponse.hits().getAt(0).fields().get("date_field").value(), equalTo((Object) dateTime));
assertThat(searchResponse.hits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) Boolean.TRUE));
assertThat(searchResponse.hits().getAt(0).fields().get("binary_field").value().toString(), equalTo(Base64.encodeBytes("testing text".getBytes("UTF8"))));
assertThat(((BytesReference) searchResponse.hits().getAt(0).fields().get("binary_field").value()).toBytesArray(), equalTo((BytesReference) new BytesArray("testing text".getBytes("UTF8"))));

}
}

0 comments on commit 0e5287f

Please sign in to comment.