Skip to content

Commit

Permalink
Merge pull request elastic#17263 from talevy/auto-convert
Browse files Browse the repository at this point in the history 
add  type conversion support to ConvertProcessor
  • Loading branch information
@talevy
talevy committed Mar 29, 2016
2 parents 9d37f45 + 2064fe3 commit 9ac3887
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 21 deletions.
30 changes: 27 additions & 3 deletions core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,23 @@ public Object convert(Object value) {
public Object convert(Object value) {
return value.toString();
}
}, AUTO {
@Override
public Object convert(Object value) {
if (!(value instanceof String)) {
return value;
}
try {
return BOOLEAN.convert(value);
} catch (IllegalArgumentException e) { }
try {
return INTEGER.convert(value);
} catch (IllegalArgumentException e) {}
try {
return FLOAT.convert(value);
} catch (IllegalArgumentException e) {}
return value;
}
};

@Override
Expand All @@ -94,18 +111,24 @@ public static Type fromString(String processorTag, String propertyName, String t
public static final String TYPE = "convert";

private final String field;
private final String targetField;
private final Type convertType;

ConvertProcessor(String tag, String field, Type convertType) {
ConvertProcessor(String tag, String field, String targetField, Type convertType) {
super(tag);
this.field = field;
this.targetField = targetField;
this.convertType = convertType;
}

String getField() {
return field;
}

String getTargetField() {
return targetField;
}

Type getConvertType() {
return convertType;
}
Expand All @@ -128,7 +151,7 @@ public void execute(IngestDocument document) {
} else {
newValue = convertType.convert(oldValue);
}
document.setFieldValue(field, newValue);
document.setFieldValue(targetField, newValue);
}

@Override
Expand All @@ -141,8 +164,9 @@ public static final class Factory extends AbstractProcessorFactory<ConvertProces
public ConvertProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
Type convertType = Type.fromString(processorTag, "type", typeProperty);
return new ConvertProcessor(processorTag, field, convertType);
return new ConvertProcessor(processorTag, field, targetField, convertType);
}
}
}
18 changes: 17 additions & 1 deletion core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ingest.core.AbstractProcessorFactory;
import org.elasticsearch.ingest.core.Processor;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;

Expand All @@ -44,6 +43,7 @@ public void testCreate() throws Exception {
ConvertProcessor convertProcessor = factory.create(config);
assertThat(convertProcessor.getTag(), equalTo(processorTag));
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field1"));
assertThat(convertProcessor.getConvertType(), equalTo(type));
}

Expand Down Expand Up @@ -88,4 +88,20 @@ public void testCreateNoTypePresent() throws Exception {
assertThat(e.getMessage(), Matchers.equalTo("[type] required property is missing"));
}
}

public void testCreateWithExplicitTargetField() throws Exception {
ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
Map<String, Object> config = new HashMap<>();
ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values());
config.put("field", "field1");
config.put("target_field", "field2");
config.put("type", type.toString());
String processorTag = randomAsciiOfLength(10);
config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
ConvertProcessor convertProcessor = factory.create(config);
assertThat(convertProcessor.getTag(), equalTo(processorTag));
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field2"));
assertThat(convertProcessor.getConvertType(), equalTo(type));
}
}
103 changes: 90 additions & 13 deletions core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,15 @@
import static org.elasticsearch.ingest.processor.ConvertProcessor.Type;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.sameInstance;

public class ConvertProcessorTests extends ESTestCase {

public void testConvertInt() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
int randomInt = randomInt();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt));
}
Expand All @@ -57,7 +58,7 @@ public void testConvertIntList() throws Exception {
expectedList.add(randomInt);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
Expand All @@ -68,7 +69,7 @@ public void testConvertIntError() throws Exception {
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);

Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
Expand All @@ -84,7 +85,7 @@ public void testConvertFloat() throws Exception {
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat);
expectedResult.put(fieldName, randomFloat);

Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Float.class), equalTo(randomFloat));
}
Expand All @@ -100,7 +101,7 @@ public void testConvertFloatList() throws Exception {
expectedList.add(randomFloat);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
Expand All @@ -111,7 +112,7 @@ public void testConvertFloatError() throws Exception {
String value = "string-" + randomAsciiOfLengthBetween(1, 10);
ingestDocument.setFieldValue(fieldName, value);

Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
Expand All @@ -129,7 +130,7 @@ public void testConvertBoolean() throws Exception {
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString);

Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, Boolean.class), equalTo(randomBoolean));
}
Expand All @@ -149,7 +150,7 @@ public void testConvertBooleanList() throws Exception {
expectedList.add(randomBoolean);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
Expand All @@ -166,7 +167,7 @@ public void testConvertBooleanError() throws Exception {
}
ingestDocument.setFieldValue(fieldName, fieldValue);

Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
Expand Down Expand Up @@ -200,7 +201,7 @@ public void testConvertString() throws Exception {
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);

Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.STRING);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedFieldValue));
}
Expand Down Expand Up @@ -236,7 +237,7 @@ public void testConvertStringList() throws Exception {
expectedList.add(randomValueString);
}
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.STRING);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
Expand All @@ -245,7 +246,7 @@ public void testConvertNonExistingField() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Type type = randomFrom(Type.values());
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, type);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
Expand All @@ -257,12 +258,88 @@ public void testConvertNonExistingField() throws Exception {
public void testConvertNullField() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
Type type = randomFrom(Type.values());
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", type);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("Field [field] is null, cannot be converted to type [" + type + "]"));
}
}

public void testAutoConvertNotString() throws Exception {
Object randomValue;
switch(randomIntBetween(0, 2)) {
case 0:
float randomFloat = randomFloat();
randomValue = randomFloat;
break;
case 1:
int randomInt = randomInt();
randomValue = randomInt;
break;
case 2:
boolean randomBoolean = randomBoolean();
randomValue = randomBoolean;
break;
default:
throw new UnsupportedOperationException();
}
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomValue));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, sameInstance(randomValue));
}

public void testAutoConvertStringNotMatched() throws Exception {
String value = "notAnIntFloatOrBool";
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", value));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, sameInstance(value));
}

public void testAutoConvertMatchBoolean() throws Exception {
boolean randomBoolean = randomBoolean();
String booleanString = Boolean.toString(randomBoolean);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", booleanString));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomBoolean));
}

public void testAutoConvertMatchInteger() throws Exception {
int randomInt = randomInt();
String randomString = Integer.toString(randomInt);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomInt));
}

public void testAutoConvertMatchFloat() throws Exception {
float randomFloat = randomFloat();
String randomString = Float.toString(randomFloat);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
processor.execute(ingestDocument);
Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
assertThat(convertedValue, equalTo(randomFloat));
}

public void testTargetField() throws Exception {
IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
int randomInt = randomInt();
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, String.valueOf(randomInt));
String targetField = fieldName + randomAsciiOfLength(5);
Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, targetField, Type.INTEGER);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt)));
assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt));

}
}
15 changes: 11 additions & 4 deletions docs/reference/ingest/ingest-node.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -668,18 +668,25 @@ Accepts a single value or an array of values.
Converts an existing field's value to a different type, such as converting a string to an integer.
If the field value is an array, all members will be converted.

The supported types include: `integer`, `float`, `string`, and `boolean`.
The supported types include: `integer`, `float`, `string`, `boolean`, and `auto`.

Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to
false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise.

Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type.
For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. And
a value of `"242.15"` will "automatically" be converted to `242.15` of type `float`. If a provided field cannot
be appropriately converted, the Convert Processor will still process successfully and leave the field value as-is. In
such a case, `target_field` will still be updated with the unconverted field value.

[[convert-options]]
.Convert Options
[options="header"]
|======
| Name | Required | Default | Description
| `field` | yes | - | The field whose value is to be converted
| `type` | yes | - | The type to convert the existing value to
| Name | Required | Default | Description
| `field` | yes | - | The field whose value is to be converted
| `target_field` | no | `field` | The field to assign the converted value to, by default `field` is updated in-place
| `type` | yes | - | The type to convert the existing value to
|======

[source,js]
Expand Down

0 comments on commit 9ac3887

Please sign in to comment.