Skip to content

Commit

Permalink
implemented semanticPropUtils class, added compatibility to operators
Browse files Browse the repository at this point in the history
  • Loading branch information
@skunert
skunert authored and StephanEwen committed May 15, 2014
1 parent 885b5e7 commit 38f8449
Show file tree
Hide file tree
Showing 10 changed files with 607 additions and 359 deletions.
282 changes: 129 additions & 153 deletions stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/FunctionAnnotation.java
View file

Large diffs are not rendered by default.

407 changes: 236 additions & 171 deletions stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java
View file

Large diffs are not rendered by default.

16 changes: 13 additions & 3 deletions ...ava/src/main/java/eu/stratosphere/api/java/operators/translation/PlanCogroupOperator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,20 @@
package eu.stratosphere.api.java.operators.translation;

import eu.stratosphere.api.common.functions.GenericCoGrouper;
import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
import eu.stratosphere.api.common.operators.base.CoGroupOperatorBase;
import eu.stratosphere.api.java.functions.CoGroupFunction;
import eu.stratosphere.api.java.functions.FunctionAnnotation;
import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.typeutils.TypeInformation;

public class PlanCogroupOperator<IN1, IN2, OUT>
import java.lang.annotation.Annotation;
import java.util.Set;

public class PlanCogroupOperator<IN1, IN2, OUT>
extends CoGroupOperatorBase<GenericCoGrouper<IN1, IN2, OUT>>
implements BinaryJavaPlanNode<IN1, IN2, OUT> {

private final TypeInformation<IN1> inType1;
private final TypeInformation<IN2> inType2;
private final TypeInformation<OUT> outType;
Expand All @@ -31,10 +37,14 @@ public PlanCogroupOperator(
CoGroupFunction<IN1, IN2, OUT> udf,
int[] keyPositions1, int[] keyPositions2, String name, TypeInformation<IN1> inType1, TypeInformation<IN2> inType2, TypeInformation<OUT> outType) {
super(udf, keyPositions1, keyPositions2, name);

this.inType1 = inType1;
this.inType2 = inType2;
this.outType = outType;

Set<Annotation> annotations = FunctionAnnotation.readDualConstantAnnotations(this.getUserCodeWrapper());
DualInputSemanticProperties dsp = SemanticPropUtil.getSemanticPropsDual(annotations, this.getInputType1(), this.getInputType2(), this.getReturnType());
this.setSemanticProperties(dsp);
}

@Override
Expand Down
22 changes: 16 additions & 6 deletions ...-java/src/main/java/eu/stratosphere/api/java/operators/translation/PlanCrossOperator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,31 +15,41 @@
package eu.stratosphere.api.java.operators.translation;

import eu.stratosphere.api.common.functions.GenericCrosser;
import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
import eu.stratosphere.api.common.operators.base.CrossOperatorBase;
import eu.stratosphere.api.java.functions.CrossFunction;
import eu.stratosphere.api.java.functions.FunctionAnnotation;
import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.typeutils.TypeInformation;

public class PlanCrossOperator<IN1, IN2, OUT>
import java.lang.annotation.Annotation;
import java.util.Set;

public class PlanCrossOperator<IN1, IN2, OUT>
extends CrossOperatorBase<GenericCrosser<IN1, IN2, OUT>>
implements BinaryJavaPlanNode<IN1, IN2, OUT>{

private final TypeInformation<IN1> inType1;
private final TypeInformation<IN2> inType2;
private final TypeInformation<OUT> outType;


public PlanCrossOperator(
CrossFunction<IN1, IN2, OUT> udf,
String name,
TypeInformation<IN1> inType1, TypeInformation<IN2> inType2, TypeInformation<OUT> outType) {
super(udf, name);

this.inType1 = inType1;
this.inType2 = inType2;
this.outType = outType;


Set<Annotation> annotations = FunctionAnnotation.readDualConstantAnnotations(this.getUserCodeWrapper());
DualInputSemanticProperties dsp = SemanticPropUtil.getSemanticPropsDual(annotations, this.getInputType1(), this.getInputType2(), this.getReturnType());
this.setSemanticProperties(dsp);

}

@Override
public TypeInformation<OUT> getReturnType() {
return this.outType;
Expand Down
21 changes: 10 additions & 11 deletions ...ava/src/main/java/eu/stratosphere/api/java/operators/translation/PlanFlatMapOperator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,37 +14,36 @@
**********************************************************************************************************************/
package eu.stratosphere.api.java.operators.translation;

import java.lang.annotation.Annotation;
import java.util.Set;

import eu.stratosphere.api.common.functions.GenericFlatMap;
import eu.stratosphere.api.common.operators.SingleInputSemanticProperties;
import eu.stratosphere.api.common.operators.base.FlatMapOperatorBase;
import eu.stratosphere.api.java.functions.FlatMapFunction;
import eu.stratosphere.api.java.functions.FunctionAnnotation;
import eu.stratosphere.api.java.functions.FunctionAnnotation.ConstantFields;
import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.typeutils.TypeInformation;

import java.lang.annotation.Annotation;
import java.util.Set;


public class PlanFlatMapOperator<T, O> extends FlatMapOperatorBase<GenericFlatMap<T, O>>
implements UnaryJavaPlanNode<T, O>
{
private final TypeInformation<T> inType;

private final TypeInformation<O> outType;


public PlanFlatMapOperator(FlatMapFunction<T, O> udf, String name, TypeInformation<T> inType, TypeInformation<O> outType) {
super(udf, name);
this.inType = inType;
this.outType = outType;

Set<Annotation> annotations = FunctionAnnotation.readSingleConstantAnnotations(this.getUserCodeWrapper());

SingleInputSemanticProperties sp = SemanticPropUtil.getSemanticPropsSingle(annotations, this.inType, this.outType);
Set<Annotation> annotations = FunctionAnnotation.readSingleConstantAnnotations(this.getUserCodeWrapper());
SingleInputSemanticProperties sp = SemanticPropUtil.getSemanticPropsSingle(annotations, this.getInputType(), this.getReturnType());
setSemanticProperties(sp);
}

@Override
public TypeInformation<O> getReturnType() {
return this.outType;
Expand Down
22 changes: 16 additions & 6 deletions ...src/main/java/eu/stratosphere/api/java/operators/translation/PlanGroupReduceOperator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,17 @@
package eu.stratosphere.api.java.operators.translation;

import eu.stratosphere.api.common.functions.GenericGroupReduce;
import eu.stratosphere.api.common.operators.SingleInputSemanticProperties;
import eu.stratosphere.api.common.operators.base.GroupReduceOperatorBase;
import eu.stratosphere.api.java.functions.FunctionAnnotation;
import eu.stratosphere.api.java.functions.GroupReduceFunction;
import eu.stratosphere.api.java.functions.GroupReduceFunction.Combinable;
import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.typeutils.TypeInformation;

import java.lang.annotation.Annotation;
import java.util.Set;

/**
*
*/
Expand All @@ -28,20 +34,24 @@ public class PlanGroupReduceOperator<IN, OUT> extends GroupReduceOperatorBase<Ge
{

private final TypeInformation<IN> inType;

private final TypeInformation<OUT> outType;
public PlanGroupReduceOperator(GroupReduceFunction<IN, OUT> udf, int[] logicalGroupingFields, String name,


public PlanGroupReduceOperator(GroupReduceFunction<IN, OUT> udf, int[] logicalGroupingFields, String name,
TypeInformation<IN> inputType, TypeInformation<OUT> outputType)
{
super(udf, logicalGroupingFields, name);

this.inType = inputType;
this.outType = outputType;
super.setCombinable(getUserCodeWrapper().getUserCodeAnnotation(Combinable.class) != null);

Set<Annotation> annotations = FunctionAnnotation.readSingleConstantAnnotations(this.getUserCodeWrapper());
SingleInputSemanticProperties sp = SemanticPropUtil.getSemanticPropsSingle(annotations, this.inType, this.outType);
setSemanticProperties(sp);
}

@Override
public TypeInformation<OUT> getReturnType() {
return this.outType;
Expand Down
18 changes: 14 additions & 4 deletions ...e-java/src/main/java/eu/stratosphere/api/java/operators/translation/PlanJoinOperator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,20 @@
package eu.stratosphere.api.java.operators.translation;

import eu.stratosphere.api.common.functions.GenericJoiner;
import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
import eu.stratosphere.api.common.operators.base.JoinOperatorBase;
import eu.stratosphere.api.java.functions.FunctionAnnotation;
import eu.stratosphere.api.java.functions.JoinFunction;
import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.typeutils.TypeInformation;

public class PlanJoinOperator<IN1, IN2, OUT>
import java.lang.annotation.Annotation;
import java.util.Set;

public class PlanJoinOperator<IN1, IN2, OUT>
extends JoinOperatorBase<GenericJoiner<IN1, IN2, OUT>>
implements BinaryJavaPlanNode<IN1, IN2, OUT> {

private final TypeInformation<IN1> inType1;
private final TypeInformation<IN2> inType2;
private final TypeInformation<OUT> outType;
Expand All @@ -31,12 +37,16 @@ public PlanJoinOperator(
JoinFunction<IN1, IN2, OUT> udf,
int[] keyPositions1, int[] keyPositions2, String name, TypeInformation<IN1> inType1, TypeInformation<IN2> inType2, TypeInformation<OUT> outType) {
super(udf, keyPositions1, keyPositions2, name);

this.inType1 = inType1;
this.inType2 = inType2;
this.outType = outType;

Set<Annotation> annotations = FunctionAnnotation.readDualConstantAnnotations(this.getUserCodeWrapper());
DualInputSemanticProperties dsp = SemanticPropUtil.getSemanticPropsDual(annotations, this.getInputType1(), this.getInputType2(), this.getReturnType());
this.setSemanticProperties(dsp);
}

@Override
public TypeInformation<OUT> getReturnType() {
return this.outType;
Expand Down
18 changes: 14 additions & 4 deletions ...re-java/src/main/java/eu/stratosphere/api/java/operators/translation/PlanMapOperator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,16 @@
package eu.stratosphere.api.java.operators.translation;

import eu.stratosphere.api.common.functions.GenericMap;
import eu.stratosphere.api.common.operators.SingleInputSemanticProperties;
import eu.stratosphere.api.common.operators.base.PlainMapOperatorBase;
import eu.stratosphere.api.java.functions.FunctionAnnotation;
import eu.stratosphere.api.java.functions.MapFunction;
import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.typeutils.TypeInformation;

import java.lang.annotation.Annotation;
import java.util.Set;

/**
*
*/
Expand All @@ -27,16 +33,20 @@ public class PlanMapOperator<T, O> extends PlainMapOperatorBase<GenericMap<T, O>
{

private final TypeInformation<T> inType;

private final TypeInformation<O> outType;


public PlanMapOperator(MapFunction<T, O> udf, String name, TypeInformation<T> inType, TypeInformation<O> outType) {
super(udf, name);
this.inType = inType;
this.outType = outType;

Set<Annotation> annotations = FunctionAnnotation.readSingleConstantAnnotations(this.getUserCodeWrapper());
SingleInputSemanticProperties sp = SemanticPropUtil.getSemanticPropsSingle(annotations, this.inType, this.outType);
setSemanticProperties(sp);
}

@Override
public TypeInformation<O> getReturnType() {
return this.outType;
Expand Down
9 changes: 8 additions & 1 deletion ...java/src/main/java/eu/stratosphere/api/java/operators/translation/PlanReduceOperator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,12 @@
import eu.stratosphere.api.common.functions.GenericReduce;
import eu.stratosphere.api.common.operators.base.ReduceOperatorBase;
import eu.stratosphere.api.java.functions.ReduceFunction;
import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.typeutils.TypeInformation;

import java.lang.annotation.Annotation;
import java.util.Set;

/**
*
*/
Expand All @@ -32,6 +36,10 @@ public class PlanReduceOperator<T> extends ReduceOperatorBase<GenericReduce<T>>
public PlanReduceOperator(ReduceFunction<T> udf, int[] logicalGroupingFields, String name, TypeInformation<T> type) {
super(udf, logicalGroupingFields, name);
this.type = type;

Set<Annotation> annotations = FunctionAnnotation.readSingleConstantAnnotations(this.getUserCodeWrapper());
SingleInputSemanticProperties sp = SemanticPropUtil.getSemanticPropsSingle(annotations, this.getInputType(), this.getReturnType());
setSemanticProperties(sp);
}


Expand All @@ -44,5 +52,4 @@ public TypeInformation<T> getReturnType() {
public TypeInformation<T> getInputType() {
return this.type;
}

}
Loading

0 comments on commit 38f8449

Please sign in to comment.