Skip to content

Commit

Permalink
ORC: Use the built-in estimate memory method (apache#4734)
Browse files Browse the repository at this point in the history
  • Loading branch information
hililiwei authored May 31, 2022
1 parent a8b494f commit 8d7d326
Showing 1 changed file with 1 addition and 15 deletions.
16 changes: 1 addition & 15 deletions orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.apache.iceberg.Metrics;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.Schema;
import org.apache.iceberg.common.DynFields;
import org.apache.iceberg.exceptions.RuntimeIOException;
import org.apache.iceberg.hadoop.HadoopOutputFile;
import org.apache.iceberg.io.FileAppender;
Expand All @@ -43,7 +42,6 @@
import org.apache.orc.StripeInformation;
import org.apache.orc.TypeDescription;
import org.apache.orc.Writer;
import org.apache.orc.impl.writer.TreeWriter;
import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -57,7 +55,6 @@ class OrcFileAppender<D> implements FileAppender<D> {
private final int batchSize;
private final OutputFile file;
private final Writer writer;
private final TreeWriter treeWriter;
private final VectorizedRowBatch batch;
private final int avgRowByteSize;
private final OrcRowWriter<D> valueWriter;
Expand Down Expand Up @@ -92,8 +89,6 @@ class OrcFileAppender<D> implements FileAppender<D> {
options.setSchema(orcSchema);
this.writer = newOrcWriter(file, options, metadata);

// TODO: Turn to access the estimateMemorySize directly after ORC 1.7.4 released with https://github.com/apache/orc/pull/1057.
this.treeWriter = treeWriterHiddenInORC();
this.valueWriter = newOrcRowWriter(schema, orcSchema, createWriterFunc);
}

Expand Down Expand Up @@ -123,10 +118,7 @@ public long length() {
return file.toInputFile().getLength();
}

Preconditions.checkNotNull(treeWriter,
"Cannot estimate length of file being written as the ORC writer's internal writer is not present");

long estimateMemory = treeWriter.estimateMemory();
long estimateMemory = writer.estimateMemory();

long dataLength = 0;
try {
Expand Down Expand Up @@ -193,10 +185,4 @@ private static <D> OrcRowWriter<D> newOrcRowWriter(Schema schema,
createWriterFunc) {
return (OrcRowWriter<D>) createWriterFunc.apply(schema, orcSchema);
}

private TreeWriter treeWriterHiddenInORC() {
DynFields.BoundField<TreeWriter> treeWriterFiled =
DynFields.builder().hiddenImpl(writer.getClass(), "treeWriter").build(writer);
return treeWriterFiled.get();
}
}

0 comments on commit 8d7d326

Please sign in to comment.