Skip to content

Commit

Permalink
JAVAFICATION: Cleanup OutputDelegatorExt
Browse files Browse the repository at this point in the history
  • Loading branch information
original-brownbear committed Jun 14, 2018
1 parent 30e055b commit 5c6e3e7
Show file tree
Hide file tree
Showing 17 changed files with 353 additions and 202 deletions.
23 changes: 19 additions & 4 deletions logstash-core/src/main/java/org/logstash/RubyUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,17 @@
import org.logstash.ackedqueue.ext.JRubyWrappedAckedQueueExt;
import org.logstash.common.AbstractDeadLetterQueueWriterExt;
import org.logstash.common.BufferedTokenizerExt;
import org.logstash.config.ir.compiler.AbstractOutputDelegatorExt;
import org.logstash.config.ir.compiler.FilterDelegatorExt;
import org.logstash.config.ir.compiler.JavaOutputDelegatorExt;
import org.logstash.config.ir.compiler.OutputDelegatorExt;
import org.logstash.config.ir.compiler.OutputStrategyExt;
import org.logstash.execution.JavaBasePipelineExt;
import org.logstash.execution.AbstractPipelineExt;
import org.logstash.execution.AbstractWrappedQueueExt;
import org.logstash.execution.ConvergeResultExt;
import org.logstash.execution.EventDispatcherExt;
import org.logstash.execution.ExecutionContextExt;
import org.logstash.execution.JavaBasePipelineExt;
import org.logstash.execution.PipelineReporterExt;
import org.logstash.execution.QueueReadClientBase;
import org.logstash.execution.ShutdownWatcherExt;
Expand Down Expand Up @@ -94,7 +96,11 @@ public final class RubyUtil {

public static final RubyClass ACKED_QUEUE_CLASS;

public static final RubyClass OUTPUT_DELEGATOR_CLASS;
public static final RubyClass ABSTRACT_OUTPUT_DELEGATOR_CLASS;

public static final RubyClass RUBY_OUTPUT_DELEGATOR_CLASS;

public static final RubyClass JAVA_OUTPUT_DELEGATOR_CLASS;

public static final RubyClass FILTER_DELEGATOR_CLASS;

Expand Down Expand Up @@ -399,8 +405,17 @@ public final class RubyUtil {
RUBY_EVENT_CLASS = setupLogstashClass(
JrubyEventExtLibrary.RubyEvent::new, JrubyEventExtLibrary.RubyEvent.class
);
OUTPUT_DELEGATOR_CLASS = setupLogstashClass(
OutputDelegatorExt::new, OutputDelegatorExt.class
ABSTRACT_OUTPUT_DELEGATOR_CLASS = LOGSTASH_MODULE.defineClassUnder(
"AbstractOutputDelegator", RUBY.getObject(),
ObjectAllocator.NOT_ALLOCATABLE_ALLOCATOR
);
ABSTRACT_OUTPUT_DELEGATOR_CLASS.defineAnnotatedMethods(AbstractOutputDelegatorExt.class);
RUBY_OUTPUT_DELEGATOR_CLASS = setupLogstashClass(
ABSTRACT_OUTPUT_DELEGATOR_CLASS, OutputDelegatorExt::new, OutputDelegatorExt.class
);
JAVA_OUTPUT_DELEGATOR_CLASS = setupLogstashClass(
ABSTRACT_OUTPUT_DELEGATOR_CLASS, JavaOutputDelegatorExt::new,
JavaOutputDelegatorExt.class
);
FILTER_DELEGATOR_CLASS = setupLogstashClass(
FilterDelegatorExt::new, FilterDelegatorExt.class
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@
import org.logstash.RubyUtil;
import org.logstash.Rubyfier;
import org.logstash.common.SourceWithMetadata;
import org.logstash.config.ir.compiler.AbstractOutputDelegatorExt;
import org.logstash.config.ir.compiler.ComputeStepSyntaxElement;
import org.logstash.config.ir.compiler.Dataset;
import org.logstash.config.ir.compiler.DatasetCompiler;
import org.logstash.config.ir.compiler.EventCondition;
import org.logstash.config.ir.compiler.FilterDelegatorExt;
import org.logstash.config.ir.compiler.OutputDelegatorExt;
import org.logstash.config.ir.compiler.RubyIntegration;
import org.logstash.config.ir.compiler.SplitDataset;
import org.logstash.config.ir.graph.IfVertex;
Expand Down Expand Up @@ -58,7 +58,7 @@ public final class CompiledPipeline {
/**
* Configured outputs.
*/
private final Map<String, OutputDelegatorExt> outputs;
private final Map<String, AbstractOutputDelegatorExt> outputs;

/**
* Parsed pipeline configuration graph.
Expand Down Expand Up @@ -103,9 +103,9 @@ public Dataset buildExecution() {
/**
* Sets up all Ruby outputs learnt from {@link PipelineIR}.
*/
private Map<String, OutputDelegatorExt> setupOutputs() {
private Map<String, AbstractOutputDelegatorExt> setupOutputs() {
final Collection<PluginVertex> outs = pipelineIR.getOutputPluginVertices();
final Map<String, OutputDelegatorExt> res = new HashMap<>(outs.size());
final Map<String, AbstractOutputDelegatorExt> res = new HashMap<>(outs.size());
outs.forEach(v -> {
final PluginDefinition def = v.getPluginDefinition();
final SourceWithMetadata source = v.getSourceWithMetadata();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package org.logstash.config.ir.compiler;

import java.util.Collection;
import java.util.concurrent.TimeUnit;
import org.jruby.Ruby;
import org.jruby.RubyArray;
import org.jruby.RubyClass;
import org.jruby.RubyObject;
import org.jruby.RubyString;
import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;
import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
import org.logstash.RubyUtil;
import org.logstash.ext.JrubyEventExtLibrary;
import org.logstash.instrument.metrics.AbstractMetricExt;
import org.logstash.instrument.metrics.AbstractNamespacedMetricExt;
import org.logstash.instrument.metrics.MetricKeys;
import org.logstash.instrument.metrics.counter.LongCounter;

@JRubyClass(name = "AbstractOutputDelegator")
public abstract class AbstractOutputDelegatorExt extends RubyObject {

private AbstractMetricExt metric;

protected AbstractNamespacedMetricExt namespacedMetric;

private IRubyObject metricEvents;

private RubyString id;

private LongCounter eventMetricOut;

private LongCounter eventMetricIn;

private LongCounter eventMetricTime;

public AbstractOutputDelegatorExt(final Ruby runtime, final RubyClass metaClass) {
super(runtime, metaClass);
}

@JRubyMethod
public IRubyObject register(final ThreadContext context) {
doRegister(context);
return context.nil;
}

@JRubyMethod(name = "do_close")
public IRubyObject doClose(final ThreadContext context) {
close(context);
return context.nil;
}

@JRubyMethod(name = "reloadable?")
public IRubyObject isReloadable(final ThreadContext context) {
return reloadable(context);
}

@JRubyMethod
public IRubyObject concurrency(final ThreadContext context) {
return getConcurrency(context);
}

@JRubyMethod(name = "config_name")
public IRubyObject configName(final ThreadContext context) {
return getConfigName(context);
}

@JRubyMethod(name = "id")
public IRubyObject getId() {
return id;
}

@JRubyMethod
public IRubyObject metric() {
return metric;
}

@JRubyMethod(name = "namespaced_metric")
public IRubyObject namespacedMetric() {
return namespacedMetric;
}

@JRubyMethod(name = "metric_events")
public IRubyObject metricEvents() {
return metricEvents;
}

@JRubyMethod(name = "multi_receive")
public IRubyObject multiReceive(final IRubyObject events) {
final RubyArray batch = (RubyArray) events;
final int count = batch.size();
eventMetricIn.increment((long) count);
final long start = System.nanoTime();
doOutput(batch);
eventMetricTime.increment(
TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS)
);
eventMetricOut.increment((long) count);
return this;
}

protected void initMetrics(final String id, final AbstractMetricExt metric) {
this.metric = metric;
final ThreadContext context = RubyUtil.RUBY.getCurrentContext();
this.id = RubyString.newString(context.runtime, id);
namespacedMetric = metric.namespace(context, context.runtime.newSymbol(id));
metricEvents = namespacedMetric.namespace(context, MetricKeys.EVENTS_KEY);
namespacedMetric.gauge(
context, MetricKeys.NAME_KEY, configName(context)
);
eventMetricOut = LongCounter.fromRubyBase(metricEvents, MetricKeys.OUT_KEY);
eventMetricIn = LongCounter.fromRubyBase(metricEvents, MetricKeys.IN_KEY);
eventMetricTime = LongCounter.fromRubyBase(
metricEvents, MetricKeys.DURATION_IN_MILLIS_KEY
);
}

protected abstract IRubyObject getConfigName(ThreadContext context);

protected abstract IRubyObject getConcurrency(ThreadContext context);

protected abstract void doOutput(Collection<JrubyEventExtLibrary.RubyEvent> batch);

protected abstract void close(ThreadContext context);

protected abstract void doRegister(ThreadContext context);

protected abstract IRubyObject reloadable(ThreadContext context);
}
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ public static Dataset terminalDataset(final Collection<Dataset> parents) {
* @return Output Dataset
*/
public static ComputeStepSyntaxElement<Dataset> outputDataset(final Collection<Dataset> parents,
final OutputDelegatorExt output, final boolean terminal) {
final AbstractOutputDelegatorExt output, final boolean terminal) {
final ClassFields fields = new ClassFields();
final Closure clearSyntax;
final Closure computeSyntax;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import org.jruby.RubyHash;
import org.jruby.RubyObject;
import org.jruby.RubyString;
import org.jruby.RubySymbol;
import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;
import org.jruby.runtime.ThreadContext;
Expand Down Expand Up @@ -47,17 +46,14 @@ public IRubyObject init(final ThreadContext context, final IRubyObject filter, f
this.filter = filter;
this.filterClass = filter.getSingletonClass().getRealClass();
final IRubyObject namespacedMetric = filter.callMethod(context, "metric");
metricEvents = namespacedMetric.callMethod(context, "namespace", RubyUtil.RUBY.newSymbol("events"));
metricEvents = namespacedMetric.callMethod(context, "namespace", MetricKeys.EVENTS_KEY);
eventMetricOut = LongCounter.fromRubyBase(metricEvents, MetricKeys.OUT_KEY);
eventMetricIn = LongCounter.fromRubyBase(metricEvents, MetricKeys.IN_KEY);
eventMetricTime = LongCounter.fromRubyBase(
metricEvents, MetricKeys.DURATION_IN_MILLIS_KEY
);
namespacedMetric.callMethod(
context, "gauge",
new IRubyObject[]{
RubySymbol.newSymbol(context.runtime, "name"), configName(context)
}
context, "gauge", new IRubyObject[]{MetricKeys.NAME_KEY, configName(context)}
);
flushes = filter.respondsTo("flush");
return this;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package org.logstash.config.ir.compiler;

import java.util.Collection;
import java.util.function.Consumer;
import org.jruby.Ruby;
import org.jruby.RubyClass;
import org.jruby.RubyString;
import org.jruby.RubySymbol;
import org.jruby.anno.JRubyClass;
import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
import org.logstash.RubyUtil;
import org.logstash.ext.JrubyEventExtLibrary;
import org.logstash.instrument.metrics.AbstractMetricExt;

@JRubyClass(name = "JavaOutputDelegator")
public final class JavaOutputDelegatorExt extends AbstractOutputDelegatorExt {

private static final RubySymbol CONCURRENCY = RubyUtil.RUBY.newSymbol("java");

private RubyString configName;

private Consumer<Collection<JrubyEventExtLibrary.RubyEvent>> outputFunction;

private Runnable closeAction;

private Runnable registerAction;

public JavaOutputDelegatorExt(final Ruby runtime, final RubyClass metaClass) {
super(runtime, metaClass);
}

public static JavaOutputDelegatorExt create(final String configName, final String id,
final AbstractMetricExt metric,
final Consumer<Collection<JrubyEventExtLibrary.RubyEvent>> outputFunction,
final Runnable closeAction, final Runnable registerAction) {
final JavaOutputDelegatorExt instance =
new JavaOutputDelegatorExt(RubyUtil.RUBY, RubyUtil.JAVA_OUTPUT_DELEGATOR_CLASS);
instance.initMetrics(id, metric);
instance.configName = RubyUtil.RUBY.newString(configName);
instance.outputFunction = outputFunction;
instance.closeAction = closeAction;
instance.registerAction = registerAction;
return instance;
}

@Override
protected IRubyObject getConfigName(final ThreadContext context) {
return configName;
}

@Override
protected IRubyObject getConcurrency(final ThreadContext context) {
return CONCURRENCY;
}

@Override
protected void doOutput(final Collection<JrubyEventExtLibrary.RubyEvent> batch) {
outputFunction.accept(batch);
}

@Override
protected void close(final ThreadContext context) {
closeAction.run();
}

@Override
protected void doRegister(final ThreadContext context) {
registerAction.run();
}

@Override
protected IRubyObject reloadable(final ThreadContext context) {
return context.tru;
}
}
Loading

0 comments on commit 5c6e3e7

Please sign in to comment.